{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\Program Files (x86)\\Python37\\lib\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "# Various torch packages\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "# torchvision\n",
    "from torchvision import datasets, transforms\n",
    "\n",
    "# ------------------------\n",
    "# get up one directory \n",
    "import sys, os\n",
    "sys.path.append(os.path.abspath('../'))\n",
    "# ------------------------\n",
    "\n",
    "# custom packages\n",
    "import models.aux_funs as maf\n",
    "import optimizers as op\n",
    "import regularizers as reg\n",
    "import train\n",
    "import math\n",
    "import utils.configuration as cf\n",
    "import utils.datasets as ud\n",
    "from utils.datasets import get_data_set, GaussianSmoothing\n",
    "from models.fully_connected import fully_connected"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# -----------------------------------------------------------------------------------\n",
    "# Fix random seed\n",
    "# -----------------------------------------------------------------------------------\n",
    "random_seed = 2\n",
    "cf.seed_torch(random_seed)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "conf_args = {#\n",
    "    # data specification\n",
    "    'data_file':\"../../Data\", 'train_split':0.95, 'data_set':\"Fashion-MNIST\", 'download':False,\n",
    "    # cuda\n",
    "    'use_cuda':True, 'num_workers':4, 'cuda_device':0, 'pin_memory':True,\n",
    "    #\n",
    "    'epochs':100,\n",
    "    # optimizer\n",
    "    'delta':1.0, 'lr':0.001, 'lamda_0':0.05, 'lamda_1':0.05, 'optim':\"AdaBreg\", 'row_group':True,\n",
    "    'reg':reg.reg_l1_l2, 'beta':0.0,\n",
    "    # model\n",
    "    'model_size':7*[28*28], 'act_fun':torch.nn.ReLU(),\n",
    "    # initialization\n",
    "    'sparse_init':0.03, 'r':[1,5,1],\n",
    "    # misc\n",
    "    'random_seed':random_seed, 'eval_acc':True, 'name':'---', 'super_type':'---'\n",
    "}\n",
    "conf = cf.Conf(**conf_args)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Define DenseNet model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class BasicBlock(nn.Module):\n",
    "    def __init__(self, in_planes, out_planes):\n",
    "        super(BasicBlock, self).__init__()\n",
    "        self.relu = nn.ReLU(inplace=True)\n",
    "        \n",
    "        \n",
    "        self.out_planes = out_planes\n",
    "        self.in_planes = in_planes\n",
    "        \n",
    "        self.bn1 = nn.BatchNorm2d(out_planes)\n",
    "        self.conv1 = nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=1,padding=1)\n",
    "        s = torch.zeros((self.in_planes,))              \n",
    "        \n",
    "    def forward(self, x):\n",
    "        out = self.relu(self.conv1(x))\n",
    "\n",
    "        return torch.cat([x, out], 1)\n",
    "    \n",
    "class DenseBlock(nn.Module):\n",
    "    def __init__(self, num_layers, in_planes, out_planes):\n",
    "        super(DenseBlock, self).__init__()\n",
    "        \n",
    "        layers = []\n",
    "        for i in range(num_layers):\n",
    "            layers.append(BasicBlock(in_planes + i*out_planes, out_planes))\n",
    "        \n",
    "        self.layer = nn.Sequential(*layers)\n",
    "\n",
    "    def forward(self, x):\n",
    "        return self.layer(x)\n",
    "\n",
    "class LinearBlock(nn.Module):\n",
    "    def __init__(self, in_size, out_size):\n",
    "        super(LinearBlock, self).__init__()\n",
    "        self.linear = nn.Linear(in_size, out_size)\n",
    "        self.act = nn.ReLU()\n",
    "    \n",
    "    def forward(self, x):\n",
    "        x = nn.Flatten()(x[:,-1,:])\n",
    "        return self.act(self.linear(x))\n",
    "\n",
    "class DenseNet(nn.Module):\n",
    "    def __init__(self, depth, planes, num_classes, mean = 0.0, std = 1.0, im_channels = 3, im_size=32):\n",
    "        super(DenseNet, self).__init__()\n",
    "        self.mean = mean\n",
    "        self.std = std\n",
    "        \n",
    "        self.depth = depth\n",
    "        self.planes = planes\n",
    "        self.num_classes = num_classes\n",
    "        \n",
    "        self.conv1 = nn.Conv2d(im_channels, planes, kernel_size=3, stride=1, padding=1, bias=False)\n",
    "        self.block1 = DenseBlock(depth, planes, planes)\n",
    "        \n",
    "        self.trans1 = BasicBlock((depth + 1) * planes, 1)\n",
    "        self.fc = LinearBlock(im_size*im_size, num_classes)\n",
    "    \n",
    "    def forward(self, x):\n",
    "        out = (x-self.mean)/self.std\n",
    "        out = self.conv1(out)\n",
    "        out = self.block1(out)\n",
    "        out = self.trans1(out)\n",
    "        return self.fc(out)     "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# -----------------------------------------------------------------------------------\n",
    "# define the model and an instance of the best model class\n",
    "# -----------------------------------------------------------------------------------\n",
    "model_kwargs = {'mean':conf.data_set_mean, 'std':conf.data_set_std, 'im_size':conf.im_shape[1],\n",
    "                'im_channels':conf.im_shape[0]}    \n",
    "\n",
    "model = DenseNet(5, 12, 10, **model_kwargs)\n",
    "    \n",
    "# sparsify\n",
    "maf.sparse_bias_uniform_(model, 0,conf.r[0])\n",
    "maf.sparse_weight_normal_(model, conf.r[1])\n",
    "maf.sparsify_(model, conf.sparse_init, row_group = conf.row_group)\n",
    "model = model.to(conf.device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# -----------------------------------------------------------------------------------\n",
    "# define the model and an instance of the best model class\n",
    "# -----------------------------------------------------------------------------------\n",
    "model_kwargs = {'mean':conf.data_set_mean, 'std':conf.data_set_std}    \n",
    "\n",
    "def init_weights(conf, model):\n",
    "    # sparsify\n",
    "    maf.sparse_bias_uniform_(model, 0, conf.r[0])\n",
    "    maf.sparse_weight_normal_(model, conf.r[1])\n",
    "    maf.sparse_weight_normal_(model, conf.r[1],ltype=nn.Conv2d)\n",
    "    maf.sparsify_(model, conf.sparse_init, ltype = nn.Conv2d, row_group = conf.row_group)\n",
    "    model = model.to(conf.device)\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# -----------------------------------------------------------------------------------\n",
    "# Optimizer\n",
    "# -----------------------------------------------------------------------------------\n",
    "def get_skips(model):\n",
    "    for m in model.modules():\n",
    "        if hasattr(m,'skips'):\n",
    "            yield m.skips\n",
    "        else:\n",
    "            continue\n",
    "\n",
    "def print_skips(model):\n",
    "    for m in model.modules():\n",
    "        if hasattr(m,'skips'):\n",
    "            print((0.001*torch.round(1000*m.skips.data).cpu()))\n",
    "            \n",
    "def skips_to_list(model):\n",
    "    skips = []\n",
    "    for m in model.modules():\n",
    "        if hasattr(m,'skips'):\n",
    "            skips.append(m.skips.data.tolist())\n",
    "    return skips\n",
    "    \n",
    "\n",
    "def init_opt(conf, model):\n",
    "    # Get access to different model parameters\n",
    "    weights_linear = maf.get_weights_linear(model)\n",
    "    weights_conv = maf.get_weights_conv(model)\n",
    "    biases = maf.get_bias(model)\n",
    "    skips = get_skips(model)\n",
    "\n",
    "    # -----------------------------------------------------------------------------------\n",
    "    # Initialize optimizer\n",
    "    # -----------------------------------------------------------------------------------\n",
    "    reg1 = conf.reg(lamda=conf.lamda_0)\n",
    "    reg2 = reg.reg_l1(lamda=conf.lamda_1)\n",
    "    reg3 = reg.reg_l1_l2_conv(lamda=conf.lamda_0)\n",
    "\n",
    "    if conf.optim == \"SGD\":\n",
    "        opt = torch.optim.SGD(model.parameters(), lr=conf.lr, momentum=conf.beta)\n",
    "    elif conf.optim == \"AdaBreg\":\n",
    "        opt = op.AdaBreg([{'params': weights_linear, 'lr' : conf.lr},\n",
    "                           {'params': weights_conv, 'lr' : conf.lr, 'reg' : reg3},\n",
    "                           {'params': biases, 'lr': conf.lr},\n",
    "                           {'params': skips, 'lr':conf.lr, 'reg':reg2}])\n",
    "\n",
    "    # learning rate scheduler\n",
    "    scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(opt, factor=0.5, patience=5,threshold=0.01)\n",
    "    \n",
    "    return opt, scheduler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "save_params = False\n",
    "if save_params:\n",
    "    conf.write_to_csv()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_loader, valid_loader, test_loader = ud.get_data_set(conf)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# History and Run Specification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# -----------------------------------------------------------------------------------\n",
    "# initalize history\n",
    "# -----------------------------------------------------------------------------------\n",
    "tracked = ['loss', 'node_sparse']\n",
    "train_hist = {}\n",
    "val_hist = {}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 0\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.6021052631578947\n",
      "Train Loss: 479.6140657365322\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.795\n",
      "Non-zero kernels: 0.09491978609625669\n",
      "Linear sparsity: 1.0\n",
      "Overall sparsity: 0.34801683549721785\n",
      "Node sparsity: [1.0]\n",
      "Regularization values per group: [0.0, 14.305777108669282, 0.0, 0.0]\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 1\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.8217017543859649\n",
      "Train Loss: 222.65356013178825\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.822\n",
      "Non-zero kernels: 0.15106951871657753\n",
      "Linear sparsity: 1.0\n",
      "Overall sparsity: 0.3884648309316593\n",
      "Node sparsity: [1.0]\n",
      "Regularization values per group: [0.0, 16.97051260471344, 0.0, 0.0]\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 2\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.8421228070175438\n",
      "Train Loss: 198.11477753520012\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.8413333333333334\n",
      "Non-zero kernels: 0.17647058823529413\n",
      "Linear sparsity: 1.0\n",
      "Overall sparsity: 0.4067627336281923\n",
      "Node sparsity: [1.0]\n",
      "Regularization values per group: [0.0, 18.669838786125187, 0.0, 0.0]\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 3\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.8542456140350877\n",
      "Train Loss: 184.61766974627972\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.8423333333333334\n",
      "Non-zero kernels: 0.18493761140819964\n",
      "Linear sparsity: 1.0\n",
      "Overall sparsity: 0.4128620345270367\n",
      "Node sparsity: [1.0]\n",
      "Regularization values per group: [0.0, 19.801439058780673, 0.0, 0.0]\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 4\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.8599824561403508\n",
      "Train Loss: 177.19984389841557\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.8486666666666667\n",
      "Non-zero kernels: 0.1929590017825312\n",
      "Linear sparsity: 1.0\n",
      "Overall sparsity: 0.41864031958909975\n",
      "Node sparsity: [1.0]\n",
      "Regularization values per group: [0.0, 20.84633814096451, 0.0, 0.0]\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 5\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.8671228070175439\n",
      "Train Loss: 169.14305187761784\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.8503333333333334\n",
      "Non-zero kernels: 0.196524064171123\n",
      "Linear sparsity: 1.0\n",
      "Overall sparsity: 0.42120844628334997\n",
      "Node sparsity: [1.0]\n",
      "Regularization values per group: [0.0, 21.80178194046021, 0.0, 0.0]\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 6\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.8710701754385964\n",
      "Train Loss: 163.6178968846798\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.8556666666666667\n",
      "Non-zero kernels: 0.2107843137254902\n",
      "Linear sparsity: 1.0\n",
      "Overall sparsity: 0.431480953060351\n",
      "Node sparsity: [1.0]\n",
      "Regularization values per group: [0.0, 22.608206748962406, 0.0, 0.0]\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 7\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.8759649122807017\n",
      "Train Loss: 156.9340548813343\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.8683333333333333\n",
      "Non-zero kernels: 0.214349376114082\n",
      "Linear sparsity: 1.0\n",
      "Overall sparsity: 0.4340490797546012\n",
      "Node sparsity: [1.0]\n",
      "Regularization values per group: [0.0, 23.32268700599671, 0.0, 0.0]\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 8\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.8785438596491228\n",
      "Train Loss: 153.72082690894604\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.8666666666666667\n",
      "Non-zero kernels: 0.23172905525846701\n",
      "Linear sparsity: 1.0\n",
      "Overall sparsity: 0.4465686973890712\n",
      "Node sparsity: [1.0]\n",
      "Regularization values per group: [0.0, 24.151674413681032, 0.0, 0.0]\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 9\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.8827368421052632\n",
      "Train Loss: 149.6146846115589\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.8666666666666667\n",
      "Non-zero kernels: 0.2375222816399287\n",
      "Linear sparsity: 1.0\n",
      "Overall sparsity: 0.45074190326722785\n",
      "Node sparsity: [1.0]\n",
      "Regularization values per group: [0.0, 24.9110062122345, 0.0, 0.0]\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 10\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.8866140350877193\n",
      "Train Loss: 145.69177976250648\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.8716666666666667\n",
      "Non-zero kernels: 0.23841354723707664\n",
      "Linear sparsity: 1.0\n",
      "Overall sparsity: 0.45138393494079043\n",
      "Node sparsity: [1.0]\n",
      "Regularization values per group: [0.0, 25.46313836574555, 0.0, 0.0]\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 11\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.8889649122807017\n",
      "Train Loss: 141.7346415221691\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.8796666666666667\n",
      "Non-zero kernels: 0.2410873440285205\n",
      "Linear sparsity: 1.0\n",
      "Overall sparsity: 0.4533100299614781\n",
      "Node sparsity: [1.0]\n",
      "Regularization values per group: [0.0, 25.997973632812503, 0.0, 0.0]\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 12\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.8915614035087719\n",
      "Train Loss: 138.74613265693188\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.878\n",
      "Non-zero kernels: 0.24910873440285206\n",
      "Linear sparsity: 1.0\n",
      "Overall sparsity: 0.45908831502354114\n",
      "Node sparsity: [1.0]\n",
      "Regularization values per group: [0.0, 26.54436178207398, 0.0, 0.0]\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 13\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.8933508771929825\n",
      "Train Loss: 135.7483097165823\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.8776666666666667\n",
      "Non-zero kernels: 0.25\n",
      "Linear sparsity: 1.0\n",
      "Overall sparsity: 0.4597303466971037\n",
      "Node sparsity: [1.0]\n",
      "Regularization values per group: [0.0, 27.13118619918824, 0.0, 0.0]\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 14\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.895578947368421\n",
      "Train Loss: 133.55801589787006\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.8756666666666667\n",
      "Non-zero kernels: 0.25133689839572193\n",
      "Linear sparsity: 1.0\n",
      "Overall sparsity: 0.4606933942074476\n",
      "Node sparsity: [1.0]\n",
      "Regularization values per group: [0.0, 27.714442420005803, 0.0, 0.0]\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 15\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.896859649122807\n",
      "Train Loss: 132.2725955992937\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.883\n",
      "Non-zero kernels: 0.2549019607843137\n",
      "Linear sparsity: 1.0\n",
      "Overall sparsity: 0.4632615209016978\n",
      "Node sparsity: [1.0]\n",
      "Regularization values per group: [0.0, 28.30118379592896, 0.0, 0.0]\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 16\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.8995263157894737\n",
      "Train Loss: 127.9973677098751\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.8893333333333333\n",
      "Non-zero kernels: 0.26158645276292336\n",
      "Linear sparsity: 1.0\n",
      "Overall sparsity: 0.468076758453417\n",
      "Node sparsity: [1.0]\n",
      "Regularization values per group: [0.0, 28.799695444107062, 0.0, 0.0]\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 17\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.9006666666666666\n",
      "Train Loss: 127.65989981591702\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.8796666666666667\n",
      "Non-zero kernels: 0.26648841354723707\n",
      "Linear sparsity: 1.0\n",
      "Overall sparsity: 0.47160793265801115\n",
      "Node sparsity: [1.0]\n",
      "Regularization values per group: [0.0, 29.401034116745, 0.0, 0.0]\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 18\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.9018245614035088\n",
      "Train Loss: 125.41302680969238\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.8826666666666667\n",
      "Non-zero kernels: 0.2736185383244207\n",
      "Linear sparsity: 1.0\n",
      "Overall sparsity: 0.47674418604651164\n",
      "Node sparsity: [1.0]\n",
      "Regularization values per group: [0.0, 30.089282226562503, 0.0, 0.0]\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 19\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.9014035087719299\n",
      "Train Loss: 123.88835780322552\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.8833333333333333\n",
      "Non-zero kernels: 0.27629233511586454\n",
      "Linear sparsity: 1.0\n",
      "Overall sparsity: 0.47867028106719933\n",
      "Node sparsity: [1.0]\n",
      "Regularization values per group: [0.0, 30.686235237121583, 0.0, 0.0]\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 20\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.9039824561403509\n",
      "Train Loss: 122.21185453236103\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.881\n",
      "Non-zero kernels: 0.27941176470588236\n",
      "Linear sparsity: 1.0\n",
      "Overall sparsity: 0.48091739192466826\n",
      "Node sparsity: [1.0]\n",
      "Regularization values per group: [0.0, 31.190980195999153, 0.0, 0.0]\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 21\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.9047719298245614\n",
      "Train Loss: 120.88993076980114\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.8816666666666667\n",
      "Non-zero kernels: 0.28342245989304815\n",
      "Linear sparsity: 1.0\n",
      "Overall sparsity: 0.4838065344556998\n",
      "Node sparsity: [1.0]\n",
      "Regularization values per group: [0.0, 31.758021211624154, 0.0, 0.0]\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 22\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.9074561403508772\n",
      "Train Loss: 118.97208927571774\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.8913333333333333\n",
      "Non-zero kernels: 0.28475935828877\n",
      "Linear sparsity: 1.0\n",
      "Overall sparsity: 0.48476958196604364\n",
      "Node sparsity: [1.0]\n",
      "Regularization values per group: [0.0, 32.322814321517946, 0.0, 0.0]\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 23\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.907578947368421\n",
      "Train Loss: 118.17654607445002\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.8933333333333333\n",
      "Non-zero kernels: 0.2887700534759358\n",
      "Linear sparsity: 1.0\n",
      "Overall sparsity: 0.4876587244970752\n",
      "Node sparsity: [1.0]\n",
      "Regularization values per group: [0.0, 32.89191956520081, 0.0, 0.0]\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 24\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.9087543859649123\n",
      "Train Loss: 116.22388423979282\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.8843333333333333\n",
      "Non-zero kernels: 0.29233511586452765\n",
      "Linear sparsity: 1.0\n",
      "Overall sparsity: 0.4902268511913254\n",
      "Node sparsity: [1.0]\n",
      "Regularization values per group: [0.0, 33.53998782634736, 0.0, 0.0]\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 25\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.9099298245614035\n",
      "Train Loss: 114.76444960385561\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.8853333333333333\n",
      "Non-zero kernels: 0.29411764705882354\n",
      "Linear sparsity: 1.0\n",
      "Overall sparsity: 0.4915109145384506\n",
      "Node sparsity: [1.0]\n",
      "Regularization values per group: [0.0, 34.202476930618296, 0.0, 0.0]\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 26\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.9111052631578948\n",
      "Train Loss: 113.35928152501583\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.8893333333333333\n",
      "Non-zero kernels: 0.29590017825311943\n",
      "Linear sparsity: 1.0\n",
      "Overall sparsity: 0.4927949778855757\n",
      "Node sparsity: [1.0]\n",
      "Regularization values per group: [0.0, 34.7957437992096, 0.0, 0.0]\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 27\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.9113508771929825\n",
      "Train Loss: 111.89447677135468\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.8903333333333333\n",
      "Non-zero kernels: 0.29590017825311943\n",
      "Linear sparsity: 1.0\n",
      "Overall sparsity: 0.4927949778855757\n",
      "Node sparsity: [1.0]\n",
      "Regularization values per group: [0.0, 35.40631613731385, 0.0, 0.0]\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 28\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.9118421052631579\n",
      "Train Loss: 111.70640065521002\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.8906666666666667\n",
      "Non-zero kernels: 0.3057040998217469\n",
      "Linear sparsity: 1.0\n",
      "Overall sparsity: 0.49985732629476387\n",
      "Node sparsity: [1.0]\n",
      "Regularization values per group: [0.0, 36.11943268775941, 0.0, 0.0]\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 29\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.9131578947368421\n",
      "Train Loss: 109.32389657944441\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.8883333333333333\n",
      "Non-zero kernels: 0.31283422459893045\n",
      "Linear sparsity: 1.0\n",
      "Overall sparsity: 0.5049222428306464\n",
      "Node sparsity: [1.0]\n",
      "Regularization values per group: [0.0, 37.016284561157235, 0.0, 0.0]\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 30\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.9136491228070175\n",
      "Train Loss: 108.73203734308481\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.893\n",
      "Non-zero kernels: 0.31327985739750447\n",
      "Linear sparsity: 1.0\n",
      "Overall sparsity: 0.5053145955200457\n",
      "Node sparsity: [1.0]\n",
      "Regularization values per group: [0.0, 37.804021954536445, 0.0, 0.0]\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 31\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.9145438596491228\n",
      "Train Loss: 107.42413487285376\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.8843333333333333\n",
      "Non-zero kernels: 0.31506238859180036\n",
      "Linear sparsity: 1.0\n",
      "Overall sparsity: 0.5065986588671708\n",
      "Node sparsity: [1.0]\n",
      "Regularization values per group: [0.0, 38.52242617607118, 0.0, 0.0]\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 32\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.9160526315789473\n",
      "Train Loss: 106.18523912131786\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.8796666666666667\n",
      "Non-zero kernels: 0.3163992869875223\n",
      "Linear sparsity: 1.0\n",
      "Overall sparsity: 0.5075617063775146\n",
      "Node sparsity: [1.0]\n",
      "Regularization values per group: [0.0, 39.283867120742805, 0.0, 0.0]\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 33\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.9156140350877193\n",
      "Train Loss: 105.53407895565033\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.894\n",
      "Non-zero kernels: 0.31595365418894833\n",
      "Linear sparsity: 1.0\n",
      "Overall sparsity: 0.5072406905407334\n",
      "Node sparsity: [1.0]\n",
      "Regularization values per group: [0.0, 39.84073991775514, 0.0, 0.0]\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 34\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.9181228070175439\n",
      "Train Loss: 104.21100955456495\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.8903333333333333\n",
      "Non-zero kernels: 0.31996434937611407\n",
      "Linear sparsity: 1.0\n",
      "Overall sparsity: 0.5101298330717648\n",
      "Node sparsity: [1.0]\n",
      "Regularization values per group: [0.0, 40.567958021163946, 0.0, 0.0]\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 35\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.9188947368421052\n",
      "Train Loss: 102.40141601860523\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.891\n",
      "Non-zero kernels: 0.321301247771836\n",
      "Linear sparsity: 1.0\n",
      "Overall sparsity: 0.5110928805821087\n",
      "Node sparsity: [1.0]\n",
      "Regularization values per group: [0.0, 41.23322596549989, 0.0, 0.0]\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 36\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.9178947368421052\n",
      "Train Loss: 102.9900024458766\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.893\n",
      "Non-zero kernels: 0.32174688057041\n",
      "Linear sparsity: 1.0\n",
      "Overall sparsity: 0.51141389641889\n",
      "Node sparsity: [1.0]\n",
      "Regularization values per group: [0.0, 41.80349700450898, 0.0, 0.0]\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 37\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.9190526315789473\n",
      "Train Loss: 101.92748426645994\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.8866666666666667\n",
      "Non-zero kernels: 0.32174688057041\n",
      "Linear sparsity: 1.0\n",
      "Overall sparsity: 0.51141389641889\n",
      "Node sparsity: [1.0]\n",
      "Regularization values per group: [0.0, 42.374577784538275, 0.0, 0.0]\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 38\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.9205263157894736\n",
      "Train Loss: 100.44658596813679\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.8956666666666667\n",
      "Non-zero kernels: 0.32174688057041\n",
      "Linear sparsity: 1.0\n",
      "Overall sparsity: 0.51141389641889\n",
      "Node sparsity: [1.0]\n",
      "Regularization values per group: [0.0, 42.8835970401764, 0.0, 0.0]\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 39\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.9208947368421052\n",
      "Train Loss: 98.79665458202362\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.887\n",
      "Non-zero kernels: 0.3226381461675579\n",
      "Linear sparsity: 1.0\n",
      "Overall sparsity: 0.5120559280924526\n",
      "Node sparsity: [1.0]\n",
      "Regularization values per group: [0.0, 43.40354290008546, 0.0, 0.0]\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 40\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.9208947368421052\n",
      "Train Loss: 98.789106734097\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.8913333333333333\n",
      "Non-zero kernels: 0.3230837789661319\n",
      "Linear sparsity: 1.0\n",
      "Overall sparsity: 0.5123769439292338\n",
      "Node sparsity: [1.0]\n",
      "Regularization values per group: [0.0, 43.96352620124817, 0.0, 0.0]\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 41\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.9213333333333333\n",
      "Train Loss: 97.67936943471432\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.8913333333333333\n",
      "Non-zero kernels: 0.3230837789661319\n",
      "Linear sparsity: 1.0\n",
      "Overall sparsity: 0.5123769439292338\n",
      "Node sparsity: [1.0]\n",
      "Regularization values per group: [0.0, 44.4576165676117, 0.0, 0.0]\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 42\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.9227017543859649\n",
      "Train Loss: 96.70330921560526\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.885\n",
      "Non-zero kernels: 0.3230837789661319\n",
      "Linear sparsity: 1.0\n",
      "Overall sparsity: 0.5123769439292338\n",
      "Node sparsity: [1.0]\n",
      "Regularization values per group: [0.0, 44.91674780845643, 0.0, 0.0]\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 43\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.9235964912280702\n",
      "Train Loss: 95.48853664845228\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.89\n",
      "Non-zero kernels: 0.32486631016042783\n",
      "Linear sparsity: 1.0\n",
      "Overall sparsity: 0.513661007276359\n",
      "Node sparsity: [1.0]\n",
      "Regularization values per group: [0.0, 45.47323687076569, 0.0, 0.0]\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 44\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.9235614035087719\n",
      "Train Loss: 95.47361582517624\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.883\n",
      "Non-zero kernels: 0.32798573975044565\n",
      "Linear sparsity: 1.0\n",
      "Overall sparsity: 0.515908118133828\n",
      "Node sparsity: [1.0]\n",
      "Regularization values per group: [0.0, 46.116224455833446, 0.0, 0.0]\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 45\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.9242982456140351\n",
      "Train Loss: 94.55944576859474\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.8886666666666667\n",
      "Non-zero kernels: 0.3293226381461676\n",
      "Linear sparsity: 1.0\n",
      "Overall sparsity: 0.5168711656441718\n",
      "Node sparsity: [1.0]\n",
      "Regularization values per group: [0.0, 46.65369036197663, 0.0, 0.0]\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 46\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.9247543859649123\n",
      "Train Loss: 92.60215925425291\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.8906666666666667\n",
      "Non-zero kernels: 0.33110516934046347\n",
      "Linear sparsity: 1.0\n",
      "Overall sparsity: 0.518155228991297\n",
      "Node sparsity: [1.0]\n",
      "Regularization values per group: [0.0, 47.1361843585968, 0.0, 0.0]\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 47\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.9261578947368421\n",
      "Train Loss: 91.39258436858654\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.8823333333333333\n",
      "Non-zero kernels: 0.33110516934046347\n",
      "Linear sparsity: 1.0\n",
      "Overall sparsity: 0.518155228991297\n",
      "Node sparsity: [1.0]\n",
      "Regularization values per group: [0.0, 47.68462550640106, 0.0, 0.0]\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 48\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.9267368421052632\n",
      "Train Loss: 91.4347620755434\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.891\n",
      "Non-zero kernels: 0.33110516934046347\n",
      "Linear sparsity: 1.0\n",
      "Overall sparsity: 0.518155228991297\n",
      "Node sparsity: [1.0]\n",
      "Regularization values per group: [0.0, 48.140287899971014, 0.0, 0.0]\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 49\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.9282982456140351\n",
      "Train Loss: 89.77467009425163\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.8923333333333333\n",
      "Non-zero kernels: 0.33110516934046347\n",
      "Linear sparsity: 1.0\n",
      "Overall sparsity: 0.518155228991297\n",
      "Node sparsity: [1.0]\n",
      "Regularization values per group: [0.0, 48.53196859359742, 0.0, 0.0]\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 50\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.9280877192982456\n",
      "Train Loss: 89.48975003510714\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.887\n",
      "Non-zero kernels: 0.33110516934046347\n",
      "Linear sparsity: 1.0\n",
      "Overall sparsity: 0.518155228991297\n",
      "Node sparsity: [1.0]\n",
      "Regularization values per group: [0.0, 48.940379047393805, 0.0, 0.0]\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 51\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.928859649122807\n",
      "Train Loss: 88.8284498155117\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.8893333333333333\n",
      "Non-zero kernels: 0.33110516934046347\n",
      "Linear sparsity: 1.0\n",
      "Overall sparsity: 0.518155228991297\n",
      "Node sparsity: [1.0]\n",
      "Regularization values per group: [0.0, 49.36951189041138, 0.0, 0.0]\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 52\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.9301754385964912\n",
      "Train Loss: 87.21500513702631\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.8866666666666667\n",
      "Non-zero kernels: 0.33110516934046347\n",
      "Linear sparsity: 1.0\n",
      "Overall sparsity: 0.518155228991297\n",
      "Node sparsity: [1.0]\n",
      "Regularization values per group: [0.0, 49.77472043037415, 0.0, 0.0]\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 53\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.9302456140350878\n",
      "Train Loss: 86.58160915970802\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.8913333333333333\n",
      "Non-zero kernels: 0.33110516934046347\n",
      "Linear sparsity: 1.0\n",
      "Overall sparsity: 0.518155228991297\n",
      "Node sparsity: [1.0]\n",
      "Regularization values per group: [0.0, 50.18593969345094, 0.0, 0.0]\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 54\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.9299298245614035\n",
      "Train Loss: 85.98969734832644\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.8863333333333333\n",
      "Non-zero kernels: 0.3315508021390374\n",
      "Linear sparsity: 1.0\n",
      "Overall sparsity: 0.5184762448280782\n",
      "Node sparsity: [1.0]\n",
      "Regularization values per group: [0.0, 50.542868256568916, 0.0, 0.0]\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 55\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.9314912280701755\n",
      "Train Loss: 85.51766414940357\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.8933333333333333\n",
      "Non-zero kernels: 0.3315508021390374\n",
      "Linear sparsity: 1.0\n",
      "Overall sparsity: 0.5184762448280782\n",
      "Node sparsity: [1.0]\n",
      "Regularization values per group: [0.0, 50.93099942207337, 0.0, 0.0]\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 56\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.9319473684210526\n",
      "Train Loss: 84.93086182326078\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.8876666666666667\n",
      "Non-zero kernels: 0.3315508021390374\n",
      "Linear sparsity: 1.0\n",
      "Overall sparsity: 0.5184762448280782\n",
      "Node sparsity: [1.0]\n",
      "Regularization values per group: [0.0, 51.33285970687867, 0.0, 0.0]\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 57\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.9325789473684211\n",
      "Train Loss: 83.60347838699818\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.8913333333333333\n",
      "Non-zero kernels: 0.3315508021390374\n",
      "Linear sparsity: 1.0\n",
      "Overall sparsity: 0.5184762448280782\n",
      "Node sparsity: [1.0]\n",
      "Regularization values per group: [0.0, 51.662696170806896, 0.0, 0.0]\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 58\n",
      "--------------------------------------------------\n",
      "Train Accuracy: 0.934\n",
      "Train Loss: 81.86947076022625\n",
      "--------------------------------------------------\n",
      "Validation Accuracy: 0.89\n",
      "Non-zero kernels: 0.3315508021390374\n",
      "Linear sparsity: 1.0\n",
      "Overall sparsity: 0.5184762448280782\n",
      "Node sparsity: [1.0]\n",
      "Regularization values per group: [0.0, 52.00349593162538, 0.0, 0.0]\n",
      "Learning rate: 0.001\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "||||||||||||||||||||||||||||||||||||||||||||||||||\n",
      "<><><><><><><><><><><><><><><><><><><><><><><><><>\n",
      "Epoch: 59\n"
     ]
    }
   ],
   "source": [
    "# -----------------------------------------------------------------------------------\n",
    "# Reinit weigts and the corresponding optimizer\n",
    "# -----------------------------------------------------------------------------------\n",
    "model = init_weights(conf, model)\n",
    "opt, scheduler = init_opt(conf, model)\n",
    "\n",
    "# -----------------------------------------------------------------------------------\n",
    "# train the model\n",
    "# -----------------------------------------------------------------------------------\n",
    "for epoch in range(conf.epochs):\n",
    "    print(25*\"<>\")\n",
    "    print(50*\"|\")\n",
    "    print(25*\"<>\")\n",
    "    print('Epoch:', epoch)\n",
    "\n",
    "    # ------------------------------------------------------------------------\n",
    "    # train step, log the accuracy and loss\n",
    "    # ------------------------------------------------------------------------\n",
    "    train_data = train.train_step(conf, model, opt, train_loader)\n",
    "\n",
    "    # update history\n",
    "    for key in tracked:\n",
    "        if key in train_data:\n",
    "            var_list = train_hist.setdefault(key, [])\n",
    "            var_list.append(train_data[key])        \n",
    "\n",
    "    # ------------------------------------------------------------------------\n",
    "    # validation step\n",
    "    val_data = train.validation_step(conf, model, opt, valid_loader)\n",
    "\n",
    "    print_skips(model)\n",
    "\n",
    "    # update history\n",
    "    for key in tracked:\n",
    "        if key in val_data:\n",
    "            var = val_data[key]\n",
    "            if isinstance(var, list):\n",
    "                for i, var_loc in enumerate(var):\n",
    "                    key_loc = key+\"_\" + str(i)\n",
    "                    var_list = val_hist.setdefault(key_loc, [])\n",
    "                    val_hist[key_loc].append(var_loc)\n",
    "            else:\n",
    "                var_list = val_hist.setdefault(key, [])\n",
    "                var_list.append(var) \n",
    "\n",
    "\n",
    "    scheduler.step(train_data['loss'])\n",
    "    print(\"Learning rate:\",opt.param_groups[0]['lr'])\n",
    "    #best_model(train_data['acc'], val_data['acc'], model=model)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  },
  "vscode": {
   "interpreter": {
    "hash": "07cecbd58096559357ff47686066ecdb82e32493890ce712e45aa4f3fdd673c4"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
